#本脚本的作用是将hdfs上的数据拉取到本地,转Gb18030,再打包压缩

#结果输出路径
merge_dir=/data/yzw/formal/result/merge
distinct_dir=/data/yzw/formal/result/distinct
parent_dir=/data/yzw/formal/result
# #创建目录
rm -rf $merge_dir $distinct_dir
mkdir $merge_dir $distinct_dir

#将hdfs结果的全路径分别写到2个文件中
echo -n "" > $merge_dir/hdfs_file_name
echo -n "" > $distinct_dir/hdfs_file_name
hdfs dfs -ls /user/hive/warehouse/ads.db/ads_get_finalindicatorame_and_zhidingunit_and_zhengshivalue | grep -v "Found" | awk -F " " '{print $8}' >>$merge_dir/hdfs_file_name
hdfs dfs -ls /user/hive/warehouse/ads.db/ads_distinct_zhibiao_nian_diqu | grep -v "Found" | awk -F " " '{print $8}'>>$distinct_dir/hdfs_file_name


for dir in $distinct_dir $merge_dir
do
    names=`cat $dir/hdfs_file_name`
    for name in $names
    #name:hdfs全路径
    do
    tail_name=`echo $name | awk -F '/' '{print $7}'`
    #取后缀，例如:000000_0
    hdfs dfs -get $name $dir/$tail_name
    echo "get $name IS FINISHED"
    echo -n "" > $dir/$tail_name.txt
    iconv -f UTF-8 -t GB18030 $dir/$tail_name -o $dir/$tail_name.txt
    #转换成GB18030格式
    echo "$name change to GB18030 IS FINISHED"
    done
    #压缩
    cd $dir
    compress_file_name=`echo $dir | awk -F '/' '{print $6}'`
    #取后缀:distinct或merge
    rm -rf $parent_dir/$compress_file_name.tar.gz
    find -name "*.txt" | xargs tar -zcvf  $parent_dir/$compress_file_name.tar.gz
    #打包时要打包的文件名只能是相对路径
    echo "$compress_file_name.tar.gz IS FINISHED"
    echo "$name IS FINISHED"
done
echo "ALL FINISHED"


